Data scientist, programmer, co-organize Montréal All-Girl Hack Night, PyLadies MTL
You can follow along with this talk at:
$ ipython notebook --pylab inline
Taken from http://donnees.ville.montreal.qc.ca/fiche/velos-comptage/ (click "Vélos - comptage")
Number of people per day on 7 bike paths (collected using sensors)
Download and unzip the zip file from this page to run this yourself.
import pandas as pd
# some display options to make figures bigger
# hide this
pd.set_option('display.max_columns', 15)
pd.set_option('display.line_width', 400)
pd.set_option('display.mpl_style', 'default')
rcParams['figure.figsize'] = (14, 7)
import matplotlib
font = {'family' : 'normal',
'weight' : 'bold',
'size' : 22}
matplotlib.rc('font', **font)
bike_data = pd.read_csv("./2012.csv")
bike_data[:5]
Date;Berri 1;Br�beuf (donn�es non disponibles);C�te-Sainte-Catherine;Maisonneuve 1;Maisonneuve 2;du Parc;Pierre-Dupuy;Rachel1;St-Urbain (donn�es non disponibles) | |
---|---|
0 | 01/01/2012;35;;0;38;51;26;10;16; |
1 | 02/01/2012;83;;1;68;153;53;6;43; |
2 | 03/01/2012;135;;2;104;248;89;3;58; |
3 | 04/01/2012;144;;1;116;318;111;8;61; |
4 | 05/01/2012;197;;2;124;330;97;13;95; |
bike_data = pd.read_csv("./2012.csv", encoding='latin1', sep=';', index_col='Date', parse_dates=True, dayfirst=True)
# Get rid of missing columns
bike_data = bike_data.dropna(axis=1)
# Only use 3 of the columns so it all fits on the screen
bike_data = bike_data[['Berri 1', u'Côte-Sainte-Catherine', 'Maisonneuve 1']]
bike_data[:5]
Berri 1 | Côte-Sainte-Catherine | Maisonneuve 1 | |
---|---|---|---|
Date | |||
2012-01-01 | 35 | 0 | 38 |
2012-01-02 | 83 | 1 | 68 |
2012-01-03 | 135 | 2 | 104 |
2012-01-04 | 144 | 1 | 116 |
2012-01-05 | 197 | 2 | 124 |
Exercise: Parse the CSVs from 2011 and earlier (warning: it's annoying)
We have a dataframe:
bike_data[:3]
Berri 1 | Côte-Sainte-Catherine | Maisonneuve 1 | |
---|---|---|---|
Date | |||
2012-01-01 | 35 | 0 | 38 |
2012-01-02 | 83 | 1 | 68 |
2012-01-03 | 135 | 2 | 104 |
bike_data.plot()
<matplotlib.axes.AxesSubplot at 0x3e59a90>
/opt/anaconda/envs/ipython-1.0.0a1/lib/python2.7/site-packages/matplotlib/font_manager.py:1224: UserWarning: findfont: Font family ['normal'] not found. Falling back to Bitstream Vera Sans (prop.get_family(), self.defaultFamily[fontext]))
bike_data.median()
Berri 1 3128.0 Côte-Sainte-Catherine 1269.0 Maisonneuve 1 2019.5 dtype: float64
bike_data.median().plot(kind='bar')
<matplotlib.axes.AxesSubplot at 0x3fd98d0>
# column slice
column_slice = bike_data[['Berri 1', 'Maisonneuve 1']]
# row slice
column_slice[:3]
Berri 1 | Maisonneuve 1 | |
---|---|---|
Date | ||
2012-01-01 | 35 | 38 |
2012-01-02 | 83 | 68 |
2012-01-03 | 135 | 104 |
column_slice.plot()
<matplotlib.axes.AxesSubplot at 0x43bcbd0>
bike_data['weekday'] = bike_data.index.weekday
bike_data.head()
Berri 1 | Côte-Sainte-Catherine | Maisonneuve 1 | weekday | |
---|---|---|---|---|
Date | ||||
2012-01-01 | 35 | 0 | 38 | 6 |
2012-01-02 | 83 | 1 | 68 | 0 |
2012-01-03 | 135 | 2 | 104 | 1 |
2012-01-04 | 144 | 1 | 116 | 2 |
2012-01-05 | 197 | 2 | 124 | 3 |
counts_by_day = bike_data.groupby('weekday').aggregate(numpy.sum)
counts_by_day
Berri 1 | Côte-Sainte-Catherine | Maisonneuve 1 | |
---|---|---|---|
weekday | |||
0 | 134298 | 60329 | 90051 |
1 | 135305 | 58708 | 92035 |
2 | 152972 | 67344 | 104891 |
3 | 160131 | 69028 | 111895 |
4 | 141771 | 56446 | 98568 |
5 | 101578 | 34018 | 62067 |
6 | 99310 | 36466 | 55324 |
counts_by_day.index = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
counts_by_day.plot()
<matplotlib.axes.AxesSubplot at 0x4a6f950>
bike_data['Berri 1'].plot()
<matplotlib.axes.AxesSubplot at 0x47cc710>
def get_weather_data(year):
url_template = "http://climate.weather.gc.ca/climateData/bulkdata_e.html?format=csv&stationID=5415&Year={year}&Month={month}&timeframe=1&submit=Download+Data"
# mctavish station: 10761, airport station: 5415
data_by_month = []
for month in range(1, 13):
url = url_template.format(year=year, month=month)
weather_data = pd.read_csv(url, skiprows=16, index_col='Date/Time', parse_dates=True).dropna(axis=1)
weather_data.columns = map(lambda x: x.replace('\xb0', ''), weather_data.columns)
weather_data = weather_data.drop(['Year', 'Day', 'Month', 'Time', 'Data Quality'], axis=1)
data_by_month.append(weather_data.dropna())
# Concatenate and drop any empty columns
return pd.concat(data_by_month).dropna(axis=1, how='all').dropna()
weather_data = get_weather_data(2012)
weather_data[:5]
Dew Point Temp (C) | Rel Hum (%) | Stn Press (kPa) | Temp (C) | Visibility (km) | Weather | Wind Spd (km/h) | |
---|---|---|---|---|---|---|---|
Date/Time | |||||||
2012-01-01 00:00:00 | -3.9 | 86 | 101.24 | -1.8 | 8.0 | Fog | 4 |
2012-01-01 01:00:00 | -3.7 | 87 | 101.24 | -1.8 | 8.0 | Fog | 4 |
2012-01-01 02:00:00 | -3.4 | 89 | 101.26 | -1.8 | 4.0 | Freezing Drizzle,Fog | 7 |
2012-01-01 03:00:00 | -3.2 | 88 | 101.27 | -1.5 | 4.0 | Freezing Drizzle,Fog | 6 |
2012-01-01 04:00:00 | -3.3 | 88 | 101.23 | -1.5 | 4.8 | Fog | 7 |
bike_data['mean temp'] = weather_data['Temp (C)'].resample('D', how='mean')
bike_data.head()
Berri 1 | Côte-Sainte-Catherine | Maisonneuve 1 | weekday | mean temp | |
---|---|---|---|---|---|
Date | |||||
2012-01-01 | 35 | 0 | 38 | 6 | 0.629167 |
2012-01-02 | 83 | 1 | 68 | 0 | 0.041667 |
2012-01-03 | 135 | 2 | 104 | 1 | -14.416667 |
2012-01-04 | 144 | 1 | 116 | 2 | -13.645833 |
2012-01-05 | 197 | 2 | 124 | 3 | -6.750000 |
bike_data[['Berri 1', 'mean temp']].plot(subplots=True)
array([<matplotlib.axes.AxesSubplot object at 0x52efed0>, <matplotlib.axes.AxesSubplot object at 0x5525a90>], dtype=object)
bike_data['Rain'] = weather_data['Weather'].str.contains('Rain').map(lambda x: int(x)).resample('D', how='mean')
bike_data[['Berri 1', 'Rain']].plot(subplots=True)
array([<matplotlib.axes.AxesSubplot object at 0x5900b10>, <matplotlib.axes.AxesSubplot object at 0x6289ed0>], dtype=object)
# Look at everything between May and September
summertime_data = bike_data['2012-05-01':'2012-09-01']
summertime_data['Berri 1'][:5] < 2500
Date 2012-05-01 True 2012-05-02 False 2012-05-03 False 2012-05-04 False 2012-05-05 False Name: Berri 1, dtype: bool
summertime_data = bike_data['2012-05-01':'2012-09-01']
bad_days = summertime_data[summertime_data['Berri 1'] < 2500]
bad_days[['Berri 1', 'Rain', 'mean temp', 'weekday']]
Berri 1 | Rain | mean temp | weekday | |
---|---|---|---|---|
Date | ||||
2012-05-01 | 1986 | 0.416667 | 9.437500 | 1 |
2012-05-08 | 1241 | 0.666667 | 12.645833 | 1 |
2012-05-22 | 2315 | 0.583333 | 18.279167 | 1 |
2012-06-02 | 943 | 0.583333 | 13.566667 | 5 |
2012-06-25 | 2245 | 0.208333 | 17.270833 | 0 |
2012-08-05 | 1864 | 0.166667 | 25.783333 | 6 |
2012-08-10 | 2414 | 0.458333 | 19.841667 | 4 |
2012-08-11 | 2453 | 0.125000 | 20.891667 | 5 |
julia = {'email': 'julia@jvns.ca', 'twitter': 'http://twitter.com/b0rk'}
print 'Email:', julia['email']
print 'Twitter:', julia['twitter']
print 'Slides: http://bit.ly/pyconca-pandas'
Email: julia@jvns.ca Twitter: http://twitter.com/b0rk Slides: http://bit.ly/pyconca-pandas